package com.dhgate.amazonCraw.pageproccess;

import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.http.HttpHost;
import org.apache.log4j.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import com.dhgate.amazonCraw.model.AmazonCategoryAttModel;
import com.google.common.collect.Lists;

/**
 * 
 * @author sunxu
 * 
 */
public class AmazonPageProcess implements PageProcessor {

	private static final Logger logger = Logger
			.getLogger(AmazonPageProcess.class);
	private static List<String> useragentList = Lists.newArrayList();
	static {
		useragentList.add("Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)");
		useragentList
				.add("Mozilla/5.0 (Windows NT 5.1; rv:22.0) Gecko/20100101 Firefox/22.0");
		useragentList
				.add("Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36");
		useragentList
				.add("Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; FDM; .NET4.0C; .NET4.0E)");
		useragentList
				.add("Mozilla/5.0 (Windows NT 6.1; rv:17.0) Gecko/20100101 Firefox/17.0");
		useragentList
				.add("Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36");
	}

	public Site getSite() {
		Site site = new Site();
		site.setUserAgent(useragentList.get(new Random().nextInt(useragentList
				.size())));
		site.setRetryTimes(5);
		site.setCycleRetryTimes(5);
		site.setSleepTime(3000);
		site.setTimeOut(30000);
		site.setDomain(".amazon.com");
		site.setHttpProxy(new HttpHost("58.215.52.159",8080));
		return site;
	}

	public void process(Page page) {
		Html html = page.getHtml();
		List<String> tempList = html.xpath(
				"//div[@class='shoppingEngineSectionHeaders']/text()").all();
		if (tempList.isEmpty()) {
			// 非列表页
			// System.out.println("非列表页" + page.getRequest().getUrl());
			List<String> urls = html.xpath("//div[@class='left_nav']//a/@href")
					.all();
			/*
			 * List<String> seeAllList =
			 * html.xpath("//div[@class='left_nav']//a") .all();
			 */
			if (!urls.isEmpty()) {
				for (String url : urls) {
					page.addTargetRequest(cleanParam(url));
				}
			}
			List<String> seeAllList = html.xpath(
					"//p[@class='seeMore']//a/@href").all();
			if (!seeAllList.isEmpty()) {
				for (String seeAllUrl : seeAllList) {
					page.addTargetRequest(cleanParam(seeAllUrl));
				}
			}
			/*
			 * if (seeAllList.size() > 0 && seeAllList != null) { /*for (String
			 * seeAllUrl : seeAllList) { if (seeAllUrl.contains("See all")) {
			 * System.out.println("含有 see all"); // 此判断是否是连接已经选择属性的如果有see all
			 * 则为变态类目直接放入see all的连接到队列 String regEx =
			 * "<a href=\"(.*?)\">See all"; Pattern pat =
			 * Pattern.compile(regEx); Matcher mat = pat.matcher(seeAllUrl); if
			 * (mat.find()) { String tempurl = mat.group(1).replaceAll("&amp;",
			 * "&"); System.out.println("See All Page :"+cleanParam(tempurl));
			 * page.addTargetRequest(cleanParam(tempurl));
			 * 
			 * } } else { System.out.println("不含 see all"); // 不含see all 放入所有 if
			 * (urls != null) { for (String url : urls) { String cleanUrl =
			 * cleanParam(url); page.addTargetRequest(cleanUrl); } } } } }
			 */
		} else {
			// 列表页
			// System.out.println("列表页" + page.getRequest().getUrl());
			Selectable url = page.getHtml().xpath(
					"div[@class='categoryRefinementsSection']/ul[@data-typeid='n']/li");
			List<String> urls = url.all();
			// System.out.println(url.get(url.size()-1));
			if (urls.size() > 0 && urls != null) {
				if (urls.get(urls.size() - 1).contains("<strong>")) {
					// 底级类目
					// cleanParam(String url)
					// System.out.println("底级类目准备解析");
					parseAtt(page);
					// 进行解析
				} else {
					// 在非底级类目中取出URL后面参数
					// System.out.println("非底级类目");
					// cleanParam(String url)
					List<String> list = page
							.getHtml()
							.xpath("div[@class='categoryRefinementsSection']/ul[@data-typeid='n']/li/a/@href")
							.all();
					for (String str : list) {
						// ***********************
						String cleanUrl = cleanParam(str);
						//String s = "http://www.amazon.com/s/ref=sr_nr_n_0?rh=n%3A1036592%2Ck%3ADisney%2Cn%3A%211036682%2Cn%3A1040664%2Cn%3A2474938011%2Cn%3A2474958011&bbn=2474938011&hidden-keywords=-Sesame+-Scooby&keywords=&ie=UTF8";
						String regEx = "http://www.amazon.com/s/ref=.*?&keywords=(.*?)&ie=UTF8";
						Pattern pat = Pattern.compile(regEx);
						Matcher mat = pat.matcher(cleanUrl);
						boolean rs = mat.find();
						if (rs) {
							System.out.println("找到了");
							if ("".equals(mat.group(1))) {
								System.out.println("没值");
								page.addTargetRequest(cleanUrl);
							}else{
								System.out.println("有值"+mat.group(1));
							}
						}else{
							System.out.println("没找到");
							System.out.println("cleanUrl="+cleanUrl);
							page.addTargetRequest(cleanUrl);
						}
						//page.addTargetRequest();
					}
				}
			}
		}
	}

	public void parseAtt(Page page) {
		Html html = page.getHtml();
		List<String> attNames = html.xpath(
				"//div[@id='refinements']//h2/text()").all();
		Document doc = page.getHtml().getDocument();
		Elements els = doc.select("div[id=refinements]").select("ul[id^=ref]");
		// System.out.println(attNames.size() + "-----" + els.size());
		List<AmazonCategoryAttModel> amazonCategoryAttModels = new ArrayList<AmazonCategoryAttModel>();
		for (int i = 1; i <= els.size() - 1; i++) {
			if ("Special Sizes".equals(attNames.get(i - 1))) {
				attNames.remove("Special Sizes");
			}
			if ("Avg. Customer Review".equals(attNames.get(i - 1))) {
				break;
			}

			String attName = attNames.get(i - 1);
			List<String> attValus = null;
			// System.out.println(attName);
			Html htmlTemp = Html.create(els.get(i).html());
			if (attName.equals("Size")) {
				// 如果属性名是SIZE单独对SIZE的属性值解析
				attValus = getSizeAtt(page, htmlTemp);
			} else {
				// 属性值不是SIZE判断是否有see more 如果有单独进行解析
				List<String> attValusNow = htmlTemp.xpath(
						"li//a//span[@class='refinementLink']/text()").all();
				// if (" See more"
				// .equals(attValusTemp.get(attValusTemp.size() - 1))) {
				// List<String> seeAllUrls = htmlTemp.xpath("li//a/@href")
				// .all();
				List<String> strs = htmlTemp
						.xpath("//li/a//span[@class='refinementLink seeMore']/span[@class='expander']/text()")
						.all();
				if (strs != null && strs.size() > 0) {
					List<String> seeAllUrls = htmlTemp.xpath("//li//a/@href")
							.all();
					// System.out.println("see more url = "+seeAllUrls.get(seeAllUrls.size()-1));
					attValus = getSeeMoreAtt(seeAllUrls
							.get(seeAllUrls.size() - 1));
					if (attValus == null) {

						if (" See more".equals(attValusNow.get(attValusNow
								.size() - 1))) {
							attValus = attValusNow;
							attValus.remove(" See more");
						}
					}
					// continue;
				} else {
					attValus = attValusNow;
				}
			}
			AmazonCategoryAttModel amazonCategoryAttModel = null;
			List<String> BreadCrumbs = getBreadCrumbs(page);
			// System.out.println(getBreadCrumbs(page));
			for (String attValue : attValus) {
				amazonCategoryAttModel = new AmazonCategoryAttModel();
				if (BreadCrumbs.size() > 0 && BreadCrumbs != null) {
					amazonCategoryAttModel.setPrimaryCategory(BreadCrumbs
							.get(0));
					if (BreadCrumbs.size() > 1) {
						amazonCategoryAttModel.setSecondaryCategory(BreadCrumbs
								.get(1));
						if (BreadCrumbs.size() > 2) {
							amazonCategoryAttModel.setThirdCategory(BreadCrumbs
									.get(2));
							if (BreadCrumbs.size() > 3) {
								amazonCategoryAttModel
										.setFourthCategory(BreadCrumbs.get(3));
								if (BreadCrumbs.size() > 4) {
									amazonCategoryAttModel
											.setFifthCategory(BreadCrumbs
													.get(4));
									if (BreadCrumbs.size() > 5) {
										amazonCategoryAttModel
												.setSixthCategory(BreadCrumbs
														.get(5));
										if (BreadCrumbs.size() > 6) {
											amazonCategoryAttModel
													.setSeventhCategory(BreadCrumbs
															.get(6));
											if (BreadCrumbs.size() > 7) {
												amazonCategoryAttModel
														.setEighthCategory(BreadCrumbs
																.get(7));
												if (BreadCrumbs.size() > 8) {
													amazonCategoryAttModel
															.setNinthCategory(BreadCrumbs
																	.get(8));
													if (BreadCrumbs.size() > 9) {
														amazonCategoryAttModel
																.setTenCategory(BreadCrumbs
																		.get(9));
													}
												}
											}
										}
									}
								}
							}
						}
					}
				}
				amazonCategoryAttModel.setBottomUrl(page.getRequest().getUrl());
				amazonCategoryAttModel.setAttName(attName);
				amazonCategoryAttModel.setAttValue(attValue);
				amazonCategoryAttModels.add(amazonCategoryAttModel);
				System.out.println(attValue);
			}
			page.putField("amazonCategoryAttModels", amazonCategoryAttModels);
			// System.out.println("======================");
		}
	}

	/**
	 * 
	 * @param url
	 * @return
	 */
	public List<String> getSeeMoreAtt(String url) {
		List<String> seeAllAtts = null;
		try {
			HttpClientDownloader downloader = new HttpClientDownloader();
			Html html = downloader.download(url, "utf-8");
			seeAllAtts = html// .regex("<span class=\"refinementLink\">(.*?)</span>").all();
					.xpath("//li/a/span[@class='refinementLink']/text()").all();
		} catch (Exception e) {
			logger.error("See more page connect time out....");
		}
		if (seeAllAtts == null) {
			try {
				System.out.println("!!!!!!!!!!!!!!Thread Sleep!!!!!!!!!!!!");
				Thread.sleep(3000);
			} catch (InterruptedException e) {
				logger.error("thread sleep exception");
			}
			return getSeeMoreAtt(url);
		} else {
			return seeAllAtts;
		}
	}

	public List<String> getSizeAtt(Page page, Html htmlTemp) {
		List<String> attValus = null;
		List<String> lis = page.getHtml()
				.xpath("//ul[@class='groupRefinement']").all();
		if (!(lis.size() == 0)) {
			attValus = page
					.getHtml()
					.xpath("//ul[@class='groupRefinement']//div[@class='buttonsprite']/a//span[@class='refinementLink']/text()")
					.all();
		} else {
			attValus = htmlTemp.xpath(
					"li//a//span[@class='refinementLink']/text()").all();
		}
		return attValus;
	}

	/**
	 * 获得面包屑
	 * 
	 * @param page
	 * @return
	 */
	public List<String> getBreadCrumbs(Page page) {
		Html html = page.getHtml();
		// List<String> breadCrumbs = html.xpath(
		// "//div//h2[@id='s-result-count']//a/text()").all();
		// String add =
		// html.xpath("//div//h2[@id='s-result-count']/span/text()")
		// .get();
		// breadCrumbs.add(add);
		// return breadCrumbs;
		List<String> breadCrumbs = html.xpath(
		// "//ul[@data-typeid='n']//a//span[2]/text()"
				"//li[@class='shoppingEngineExpand']//a//span[2]/text()").all();
		String add = html.xpath(
				"//div[@class='categoryRefinementsSection']//strong/text()")
				.get();
		breadCrumbs.add(add);
		return breadCrumbs;
	}

	public String cleanParam(String url) {
		String cleanUrl = null;
		// 清理URL参数
		if (url.contains("&pf_rd_m")) {
			cleanUrl = url.substring(0, url.indexOf("&pf_rd_m"));
		}
		if (url.contains("&qid")) {
			cleanUrl = url.substring(0, url.indexOf("&qid"));
		}
		if (url.contains("&url=node")) {
			cleanUrl = url.substring(0, url.indexOf("&url=node"));
		}
		return cleanUrl;
	}
}